execute_path=$(pwd)
echo ${execute_path}

export DEPLOY_MODE=0
export CONFIG_PATH=$1
export EXP_TYPE=$2
export EXP_NAME=$3
export RANK_SIZE=$4
RANK_START=$5

export TEMP_PATH=/home/liupan/temp/startrank${RANK_START}
export LOCAL_DEFAULT_PATH=$TEMP_PATH
#export RANK_TABLE_FILE=${execute_path}/hccl/rank_table_${RANK_SIZE}pcs.json
export RANK_TABLE_FILE=/home/liupan/WingsofPanda/WP-GeneralPurposeMedicalFoundationModels/hccl/hccl_8p_01234567_127.0.1.2.json


DEVICE_START=0

source /usr/local/Ascend/driver/bin/setenv.bash
rm -rf $TEMP_PATH/WP-GeneralPurposeMedicalFoundationModels
rsync -rv --exclude-from=.copyignore $execute_path $TEMP_PATH
cd $TEMP_PATH/WP-GeneralPurposeMedicalFoundationModels || exit

#export LOG_PATH=/home/liupan/output/${EXP_TYPE}_${EXP_NAME}
export LOG_PATH=/home/liupan/output/${EXP_TYPE}_${EXP_NAME}
mkdir $LOG_PATH

for((i=0;i<=7;i++));
do
  export RANK_ID=$[i+RANK_START]
  export DEVICE_ID=$[i+DEVICE_START]
  echo "start training for rank $RANK_ID, device $DEVICE_ID"
  python3 train.py --config $CONFIG_PATH --exp_type $EXP_TYPE --exp_name $EXP_NAME --output_dir $LOG_PATH 2>&1 | tee $LOG_PATH/print_rank_${RANK_ID}.log &
done
cd $execute_path || exit
